#!/usr/bin/python
# -*- coding: utf-8 -*-
#####################################
#author:        anonymous0502
#description:   scan threads list of chinaunix forum
#date:          2012.07.05
#####################################

import sys
import urllib.request
import re
import subprocess as sp
import json
import threading
import queue
import time
from PySide.QtCore import *
from PySide.QtGui import *

from pprint import pprint


def myprint(obj, end='\n'):
    sys.stdout.write(str(obj) + end)


class MySignal(QObject):
    sig = Signal(str)


class cuwindow(QWidget):
    def __init__(self):
        QWidget.__init__(self)
        self.initdata()
        self.initgui()
        self.connsigtoslot()


    def initgui(self):
        self.setGeometry(*self.windowsize)
        self.setWindowTitle('cu')

        self.btn_del=QPushButton('delete',self)
        self.btn_save=QPushButton('save',self)
        self.btn_load=QPushButton('load',self)
        self.btn_get=QPushButton('get threads list', self)
        self.cbb_board=QComboBox()
        self.cbb_board.addItems([x[1] for x in self.boards])
        self.cbb_board.setCurrentIndex(1)
        self.labelreply=QLabel('回复数')
        self.ltxt_reply=QLineEdit(self.initdatas['reply'],self)
        self.labelbeginpages=QLabel('起始页')
        self.ltxt_beginpages=QLineEdit(self.initdatas['beginpages'],self)
        self.labelendpages=QLabel('终止页')
        self.ltxt_endpages=QLineEdit(self.initdatas['endpages'],self)
        self.view_result=QTableWidget()
        self.settbvheaders()

        self.view_result.setEditTriggers(QAbstractItemView.NoEditTriggers)
        self.view_result.setAlternatingRowColors(True)
        self.view_result.setSelectionMode(QAbstractItemView.SingleSelection)
        self.view_result.setSelectionBehavior(QAbstractItemView.SelectRows)
        self.view_result.setSortingEnabled(True)

        layout = QGridLayout()
        layout.addWidget(self.btn_del,0,0,1,1)
        layout.addWidget(self.btn_save,0,1,1,1)
        layout.addWidget(self.btn_load,0,2,1,1)
        layout.addWidget(self.btn_get,0,3,1,1)
        layout.addWidget(self.cbb_board,0,4,1,1)
        layout.addWidget(self.labelreply,0,5,1,1)
        layout.addWidget(self.ltxt_reply,0,6,1,1)
        layout.addWidget(self.labelbeginpages,0,7,1,1)
        layout.addWidget(self.ltxt_beginpages,0,8,1,1)
        layout.addWidget(self.labelendpages,0,9,1,1)
        layout.addWidget(self.ltxt_endpages,0,10,1,1)
        layout.addWidget(self.view_result,1,0,11,11)

        self.setLayout(layout)

        style = "QWidget { background-color: #92de8b }"
        self.setStyleSheet(style)


    def settbvheaders(self):
        self.view_result.setColumnCount(self.tbvcols)
        for i in range(len(self.tbvheaders)):
            self.view_result.setHorizontalHeaderItem(i,QTableWidgetItem(self.tbvheaders[i]))


    def initdata(self):
        self.boards=(
            (216,'新手园地'),
            (217,'系统管理'),
            (55,'python'),
        )
        self.initdatas={
            'reply':'0',
            'beginpages':'1',
            'endpages':'100',
        }
        self.windowsize=(100,200,800,500)
        self.tbvcols=5
        self.tbvheaders=('url','thread id','thread title','date','reply')
        self.dumpfile=r'c:/temp/threadslist.txt'
        self.numberofthreads=5
        self.lock1=threading.Lock()
        self.qurl=queue.Queue()
        self.qresult=queue.Queue()
        #保存线程对象的列表
        self.threads=[]
        #保存获取到的帖子信息的列表
        self.threadslist=[]
        self.errurl=[]


    def connsigtoslot(self):
        self.btn_get.clicked.connect(self.callget)
        self.view_result.cellDoubleClicked.connect(self.doubleclickcell)
        self.btn_save.clicked.connect(self.callsave)
        self.btn_load.clicked.connect(self.callload)
        self.btn_del.clicked.connect(self.calldel)


    def callget(self):
        boardid=self.boards[self.cbb_board.currentIndex()][0]
        replycount=self.ltxt_reply.text()
        beginpage=int(self.ltxt_beginpages.text())
        endpage=int(self.ltxt_endpages.text())

        totalpages=endpage-beginpage+1
        begintime=time.time()
        endtime=0
        rowcount=0

        for i in range(beginpage,endpage+1):
            url=r'http://bbs.chinaunix.net/forum-%d-%d.html' % (boardid,i)
            self.qurl.put(url)

        for i in range(self.numberofthreads):
            th=threading.Thread(target=self.getthreads,args=(boardid,replycount,self.lock1),name='getthreadlist%d' % i)
            th.start()
            self.threads.append(th)

        while True:
            activethreads=[x for x in self.threads if x.isAlive()]
            if len(activethreads)<=0 and self.qurl.qsize()<=0:
                break

            while self.qresult.qsize()>0:
                row=self.qresult.get()
                if len(self.threadslist)>0:
                    threadidlist=[x[1].strip() for x in self.threadslist]
                else:
                    threadidlist=[]

                if row[1].strip() not in threadidlist:
                    self.threadslist.append(row)
                    self.view_result.insertRow(rowcount)
                    for col in range(len(row)):
                        self.view_result.setItem(rowcount,col,QTableWidgetItem(str(row[col])))
                        QCoreApplication.processEvents()
                    rowcount+=1

            info='progress:%d/%d' % (totalpages-self.qurl.qsize(),totalpages)
            myprint(info)

            time.sleep(0.5)

        self.view_result.resizeColumnsToContents()
        self.view_result.resizeRowsToContents()

        endtime=time.time()
        totaltime=int(endtime-begintime)
        info='scan %d pages consume time: %d seconds=%.2f minutes.' % (totalpages,totaltime,totaltime/60.0)
        myprint(info)


    def getthreads(self,boardid,replycount,lock):
        restr=r'<a href="(thread-(\d+)-\d+-\d+.html)" onclick="atarget\(this\)" class="xst" >(.*)</a>|<em><span>(\d{4}-\d{2}-\d{2})</span></em>|<a href="thread-\d+-\d+-\d+.html" class="xi2">(\d+)</a>'
        pattern=re.compile(restr,re.M)
        sleeptime=0.5

        while True:
            with self.lock1:
                if self.qurl.qsize()>0:
                    url=self.qurl.get()
                else:
                    break

            try:
                html=self.gethtml(url)
            except Exception as ex:
                myprint(ex)
                continue

            #for debug,read from/write to local file.
            #open('c:/temp/cu.txt','w').write(html)
            #html=open('c:/temp/cu.txt','r').read()

            t=pattern.findall(html)
            if len(t)==0:
                continue

            t2=[]
            #result=[]
            for a,b,c,d,e in t:
                #link
                if a.strip():
                    t2.append(r'http://bbs.chinaunix.net/'+a)
                #thread id
                if b.strip():
                    t2.append(b)
                #title
                if c.strip():
                    t2.append(c)
                #time
                if d.strip():
                    t2.append(d)
                #reply
                if e.strip():
                    t2.append(e)

            for j in range(0,len(t2),self.tbvcols):
                #result.append(t2[j:j+self.tbvcols])
                row=t2[j:j+self.tbvcols]
                if row[-1].strip()==replycount:
                    if len(row)!=self.tbvcols:
                        continue
                    with self.lock1:
                        self.qresult.put(row)

            time.sleep(sleeptime)


    def gethtml(self,url):
        response=urllib.request.urlopen(url)
        html=response.read().decode('gbk')
        return html


    def doubleclickcell(self,row,col):
        if col==2:
            totalrowcount=self.view_result.rowCount()
            totalcolcount=self.view_result.columnCount()
            myprint('row=%d col=%d' % (totalrowcount,totalcolcount))
        if col!=0:
            return

        url=self.view_result.currentItem().text()
        cmd=r'"C:\Users\zero\AppData\Local\Google\Chrome\Application\chrome.exe" %s' % url
        sp.Popen(cmd)


    def callsave(self):
        self.save()


    def save(self):
        with open(self.dumpfile,'w') as f:
            json.dump(self.threadslist,f)
        myprint('save to %s finished.' % self.dumpfile)


    def callload(self):
        self.load()


    def load(self):
        with open(self.dumpfile,'r') as f:
            self.threadslist=json.load(f)
        myprint('from %s load data finished.' % self.dumpfile)
        rowcount=0
        self.cleartbv()
        self.settbvheaders()
        for row in self.threadslist:
            self.view_result.insertRow(rowcount)
            for col in range(len(row)):
                self.view_result.setItem(rowcount,col,QTableWidgetItem(str(row[col])))
            rowcount+=1
        self.view_result.resizeColumnsToContents()
        self.view_result.resizeRowsToContents()


    def calldel(self):
        self.delete()


    def delete(self):
        currow=self.view_result.currentRow()
        myprint(currow)
        del self.threadslist[currow]
        self.view_result.removeRow(currow)


    def cleartbv(self):
        totalrowcount=self.view_result.rowCount()
        totalcolcount=self.view_result.columnCount()
        #myprint('clear tablewidget all rows:%d %d' % (totalrowcount,totalcolcount))
        for i in range(totalrowcount-1,-1,-1):
            self.view_result.removeRow(i)
        for j in range(totalcolcount-1,-1,-1):
            self.view_result.removeColumn(j)


def main():
    app = QApplication(sys.argv)
    window=cuwindow()
    window.show()
    app.exec_()


if __name__ == '__main__':
    main()
    #myprint('done')